{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "0430.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yananma/5_programs_per_day/blob/master/0430.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fCnbT2vraU11",
        "colab_type": "text"
      },
      "source": [
        "## 3.13 丢弃法"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TbToD5gTauSU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !pip install mxnet d2lzh"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sED6LNnCms6h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import d2lzh as d2l \n",
        "from mxnet import autograd, gluon, init, nd \n",
        "from mxnet.gluon import loss as gloss, nn \n",
        "\n",
        "def dropout(X, drop_prob):\n",
        "    assert 0 <= drop_prob <= 1 \n",
        "    keep_prob = 1 - drop_prob\n",
        "    if keep_prob == 0:\n",
        "        return X.zeros_like()\n",
        "    mask = nd.random.uniform(0, 1, X.shape) < keep_prob \n",
        "    return mask * X / keep_prob"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kwh-8_U3u6mT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "46b5b562-f161-4b6a-ed0b-6e8cacf99edf"
      },
      "source": [
        "X = nd.arange(16).reshape((2, 8))\n",
        "dropout(X, 0)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[ 0.  1.  2.  3.  4.  5.  6.  7.]\n",
              " [ 8.  9. 10. 11. 12. 13. 14. 15.]]\n",
              "<NDArray 2x8 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3mwiQi7cvFtO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        },
        "outputId": "d174d269-ee56-406d-c33c-59e47e9d775e"
      },
      "source": [
        "my_mask = nd.random.uniform(0, 1, X.shape)\n",
        "my_mask"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[0.03067934 0.7395002  0.81692976 0.11655104 0.27300504 0.8716068\n",
              "  0.08849455 0.4784922 ]\n",
              " [0.8272113  0.6626148  0.3926571  0.33689874 0.49089834 0.6919619\n",
              "  0.5759835  0.4859751 ]]\n",
              "<NDArray 2x8 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eGuKDPk5xeKe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "6101f38d-83c4-42a2-81e1-81474874cfe0"
      },
      "source": [
        "my_mask = nd.random.uniform(0, 1, X.shape) < 0.5 \n",
        "my_mask"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[0. 1. 1. 1. 1. 0. 1. 1.]\n",
              " [0. 1. 0. 1. 1. 1. 1. 1.]]\n",
              "<NDArray 2x8 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oH3AujhNxxRO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "78a4deae-9a55-42c6-bc2e-feee49fcafcb"
      },
      "source": [
        "my_mask = nd.random.uniform(0, 1, X.shape) < 0.8\n",
        "my_mask"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[1. 1. 1. 0. 1. 1. 1. 1.]\n",
              " [1. 1. 1. 1. 1. 0. 1. 1.]]\n",
              "<NDArray 2x8 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eRZVoUv-x4th",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "1a05d7f9-37f6-4e22-fe0a-04a288e515d1"
      },
      "source": [
        "my_mask = nd.random.uniform(0, 1, X.shape) < 0.2 \n",
        "my_mask"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[0. 0. 0. 0. 0. 0. 0. 0.]\n",
              " [0. 0. 0. 0. 0. 0. 0. 0.]]\n",
              "<NDArray 2x8 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fqi3fRZpx8eX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "21ece17f-9631-4649-af2d-b23bfe1ee4c7"
      },
      "source": [
        "dropout(X, 0.5)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[ 0.  0.  0.  6.  0. 10. 12.  0.]\n",
              " [ 0. 18. 20.  0.  0. 26.  0.  0.]]\n",
              "<NDArray 2x8 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hdox0YSb07pu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "1d4e9cfa-2105-4cca-af94-2f9f1cb4789b"
      },
      "source": [
        "dropout(X, 1)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\n",
              "[[0. 0. 0. 0. 0. 0. 0. 0.]\n",
              " [0. 0. 0. 0. 0. 0. 0. 0.]]\n",
              "<NDArray 2x8 @cpu(0)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VA0bBnqX0_O3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "num_inputs, num_outputs, num_hiddens1, num_hiddens2 = 784, 10, 256, 256\n",
        "\n",
        "W1 = nd.random.normal(scale=0.01, shape=(num_inputs, num_hiddens1))\n",
        "b1 = nd.zeros(num_hiddens1)\n",
        "W2 = nd.random.normal(scale=0.01, shape=(num_hiddens1, num_hiddens2))\n",
        "b2 = nd.zeros(num_hiddens2)\n",
        "W3 = nd.random.normal(scale=0.01, shape=(num_hiddens2, num_outputs))\n",
        "b3 = nd.zeros(num_outputs)\n",
        "\n",
        "params = [W1, b1, W2, b2, W3, b3]\n",
        "for param in params:\n",
        "    param.attach_grad()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Me5EABfb2bDn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "drop_prob1, drop_prob2 = 0.2, 0.5 \n",
        "\n",
        "def net(X):\n",
        "    X = X.reshape((-1, num_inputs))\n",
        "    H1 = (nd.dot(X, W1) + b1).relu()\n",
        "    if autograd.is_training():\n",
        "        H1 = dropout(H1, drop_prob2)\n",
        "    H2 = (nd.dot(H1, W2) + b2).relu()\n",
        "    if autograd.is_training():\n",
        "        H2 = dropout(H2, drop_prob1)\n",
        "    return nd.dot(H2, W3) + b3 "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u4bTvNi131oH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        },
        "outputId": "92255f7b-1920-4c51-c8c4-e3ce5214b178"
      },
      "source": [
        "num_epochs, lr, batch_size = 5, 0.5, 256 \n",
        "loss = gloss.SoftmaxCrossEntropyLoss()\n",
        "train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)\n",
        "d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, params, lr)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch 1, loss 1.1191, train acc 0.564, test acc 0.769\n",
            "epoch 2, loss 0.5894, train acc 0.779, test acc 0.815\n",
            "epoch 3, loss 0.5083, train acc 0.813, test acc 0.831\n",
            "epoch 4, loss 0.4661, train acc 0.829, test acc 0.849\n",
            "epoch 5, loss 0.4392, train acc 0.840, test acc 0.855\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p1Wo57m-FSOE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "net = nn.Sequential()\n",
        "net.add(nn.Dense(256, activation='relu'), \n",
        "    nn.Dropout(drop_prob1), \n",
        "    nn.Dense(256, activation='relu'), \n",
        "    nn.Dropout(drop_prob2), \n",
        "    nn.Dense(10))\n",
        "net.initialize(init.Normal(sigma=0.01))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PA0NkvQISq0-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101
        },
        "outputId": "3865a8db-c863-48e2-8812-1a8a2c7b1873"
      },
      "source": [
        "trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': lr})\n",
        "d2l.train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, None, None, trainer)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch 1, loss 1.1179, train acc 0.561, test acc 0.785\n",
            "epoch 2, loss 0.5754, train acc 0.787, test acc 0.829\n",
            "epoch 3, loss 0.4867, train acc 0.823, test acc 0.840\n",
            "epoch 4, loss 0.4409, train acc 0.838, test acc 0.849\n",
            "epoch 5, loss 0.4194, train acc 0.848, test acc 0.860\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}